{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "26b3433ef9e24cdf",
   "metadata": {},
   "source": [
    "# ColPali"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ad805457f501a223",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T06:14:11.898553Z",
     "start_time": "2025-07-08T06:14:00.210102Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ProgramData\\miniconda3\\envs\\colpali\\Lib\\site-packages\\requests\\__init__.py:86: RequestsDependencyWarning: Unable to find acceptable character detection dependency (chardet or charset_normalizer).\n",
      "  warnings.warn(\n",
      "D:\\ProgramData\\miniconda3\\envs\\colpali\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from PIL import Image\n",
    "from transformers.utils.import_utils import is_flash_attn_2_available\n",
    "\n",
    "from colpali_engine.models import ColQwen2, ColQwen2Processor, ColPali, ColPaliProcessor\n",
    "import os\n",
    "from colpali_engine.interpretability import get_similarity_maps_from_embeddings\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3adbe8df71fcfb8c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T06:14:13.349152Z",
     "start_time": "2025-07-08T06:14:13.345496Z"
    }
   },
   "outputs": [],
   "source": [
    "# 禁用所有网络请求，强制使用本地文件（保持不变）\n",
    "os.environ[\"TRANSFORMERS_OFFLINE\"] = \"1\"\n",
    "os.environ[\"HF_DATASETS_OFFLINE\"] = \"1\"\n",
    "os.environ[\"HF_HUB_OFFLINE\"] = \"1\"  # 新增，彻底禁用huggingface hub网络请求\n",
    "\n",
    "# 本地模型路径（替换为colpali的本地目录，无需基础模型路径）\n",
    "local_model_path = \"./models/colpali-v1.3-merged\"  # 确保该目录下有从vidore/colpali下载的所有文件\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36e99c1776c87486",
   "metadata": {},
   "source": [
    "## 模型加载"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9635b95bdf9f89b1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T06:17:26.902706Z",
     "start_time": "2025-07-08T06:14:15.294451Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading checkpoint shards: 100%|██████████| 2/2 [00:00<00:00, 14.25it/s]\n",
      "Some weights of the model checkpoint at ./models/colpali-v1.3-merged were not used when initializing ColPali: ['model.language_model.model.embed_tokens.weight', 'model.language_model.model.layers.0.input_layernorm.weight', 'model.language_model.model.layers.0.mlp.down_proj.weight', 'model.language_model.model.layers.0.mlp.gate_proj.weight', 'model.language_model.model.layers.0.mlp.up_proj.weight', 'model.language_model.model.layers.0.post_attention_layernorm.weight', 'model.language_model.model.layers.0.self_attn.k_proj.weight', 'model.language_model.model.layers.0.self_attn.o_proj.weight', 'model.language_model.model.layers.0.self_attn.q_proj.weight', 'model.language_model.model.layers.0.self_attn.v_proj.weight', 'model.language_model.model.layers.1.input_layernorm.weight', 'model.language_model.model.layers.1.mlp.down_proj.weight', 'model.language_model.model.layers.1.mlp.gate_proj.weight', 'model.language_model.model.layers.1.mlp.up_proj.weight', 'model.language_model.model.layers.1.post_attention_layernorm.weight', 'model.language_model.model.layers.1.self_attn.k_proj.weight', 'model.language_model.model.layers.1.self_attn.o_proj.weight', 'model.language_model.model.layers.1.self_attn.q_proj.weight', 'model.language_model.model.layers.1.self_attn.v_proj.weight', 'model.language_model.model.layers.10.input_layernorm.weight', 'model.language_model.model.layers.10.mlp.down_proj.weight', 'model.language_model.model.layers.10.mlp.gate_proj.weight', 'model.language_model.model.layers.10.mlp.up_proj.weight', 'model.language_model.model.layers.10.post_attention_layernorm.weight', 'model.language_model.model.layers.10.self_attn.k_proj.weight', 'model.language_model.model.layers.10.self_attn.o_proj.weight', 'model.language_model.model.layers.10.self_attn.q_proj.weight', 'model.language_model.model.layers.10.self_attn.v_proj.weight', 'model.language_model.model.layers.11.input_layernorm.weight', 'model.language_model.model.layers.11.mlp.down_proj.weight', 'model.language_model.model.layers.11.mlp.gate_proj.weight', 'model.language_model.model.layers.11.mlp.up_proj.weight', 'model.language_model.model.layers.11.post_attention_layernorm.weight', 'model.language_model.model.layers.11.self_attn.k_proj.weight', 'model.language_model.model.layers.11.self_attn.o_proj.weight', 'model.language_model.model.layers.11.self_attn.q_proj.weight', 'model.language_model.model.layers.11.self_attn.v_proj.weight', 'model.language_model.model.layers.12.input_layernorm.weight', 'model.language_model.model.layers.12.mlp.down_proj.weight', 'model.language_model.model.layers.12.mlp.gate_proj.weight', 'model.language_model.model.layers.12.mlp.up_proj.weight', 'model.language_model.model.layers.12.post_attention_layernorm.weight', 'model.language_model.model.layers.12.self_attn.k_proj.weight', 'model.language_model.model.layers.12.self_attn.o_proj.weight', 'model.language_model.model.layers.12.self_attn.q_proj.weight', 'model.language_model.model.layers.12.self_attn.v_proj.weight', 'model.language_model.model.layers.13.input_layernorm.weight', 'model.language_model.model.layers.13.mlp.down_proj.weight', 'model.language_model.model.layers.13.mlp.gate_proj.weight', 'model.language_model.model.layers.13.mlp.up_proj.weight', 'model.language_model.model.layers.13.post_attention_layernorm.weight', 'model.language_model.model.layers.13.self_attn.k_proj.weight', 'model.language_model.model.layers.13.self_attn.o_proj.weight', 'model.language_model.model.layers.13.self_attn.q_proj.weight', 'model.language_model.model.layers.13.self_attn.v_proj.weight', 'model.language_model.model.layers.14.input_layernorm.weight', 'model.language_model.model.layers.14.mlp.down_proj.weight', 'model.language_model.model.layers.14.mlp.gate_proj.weight', 'model.language_model.model.layers.14.mlp.up_proj.weight', 'model.language_model.model.layers.14.post_attention_layernorm.weight', 'model.language_model.model.layers.14.self_attn.k_proj.weight', 'model.language_model.model.layers.14.self_attn.o_proj.weight', 'model.language_model.model.layers.14.self_attn.q_proj.weight', 'model.language_model.model.layers.14.self_attn.v_proj.weight', 'model.language_model.model.layers.15.input_layernorm.weight', 'model.language_model.model.layers.15.mlp.down_proj.weight', 'model.language_model.model.layers.15.mlp.gate_proj.weight', 'model.language_model.model.layers.15.mlp.up_proj.weight', 'model.language_model.model.layers.15.post_attention_layernorm.weight', 'model.language_model.model.layers.15.self_attn.k_proj.weight', 'model.language_model.model.layers.15.self_attn.o_proj.weight', 'model.language_model.model.layers.15.self_attn.q_proj.weight', 'model.language_model.model.layers.15.self_attn.v_proj.weight', 'model.language_model.model.layers.16.input_layernorm.weight', 'model.language_model.model.layers.16.mlp.down_proj.weight', 'model.language_model.model.layers.16.mlp.gate_proj.weight', 'model.language_model.model.layers.16.mlp.up_proj.weight', 'model.language_model.model.layers.16.post_attention_layernorm.weight', 'model.language_model.model.layers.16.self_attn.k_proj.weight', 'model.language_model.model.layers.16.self_attn.o_proj.weight', 'model.language_model.model.layers.16.self_attn.q_proj.weight', 'model.language_model.model.layers.16.self_attn.v_proj.weight', 'model.language_model.model.layers.17.input_layernorm.weight', 'model.language_model.model.layers.17.mlp.down_proj.weight', 'model.language_model.model.layers.17.mlp.gate_proj.weight', 'model.language_model.model.layers.17.mlp.up_proj.weight', 'model.language_model.model.layers.17.post_attention_layernorm.weight', 'model.language_model.model.layers.17.self_attn.k_proj.weight', 'model.language_model.model.layers.17.self_attn.o_proj.weight', 'model.language_model.model.layers.17.self_attn.q_proj.weight', 'model.language_model.model.layers.17.self_attn.v_proj.weight', 'model.language_model.model.layers.2.input_layernorm.weight', 'model.language_model.model.layers.2.mlp.down_proj.weight', 'model.language_model.model.layers.2.mlp.gate_proj.weight', 'model.language_model.model.layers.2.mlp.up_proj.weight', 'model.language_model.model.layers.2.post_attention_layernorm.weight', 'model.language_model.model.layers.2.self_attn.k_proj.weight', 'model.language_model.model.layers.2.self_attn.o_proj.weight', 'model.language_model.model.layers.2.self_attn.q_proj.weight', 'model.language_model.model.layers.2.self_attn.v_proj.weight', 'model.language_model.model.layers.3.input_layernorm.weight', 'model.language_model.model.layers.3.mlp.down_proj.weight', 'model.language_model.model.layers.3.mlp.gate_proj.weight', 'model.language_model.model.layers.3.mlp.up_proj.weight', 'model.language_model.model.layers.3.post_attention_layernorm.weight', 'model.language_model.model.layers.3.self_attn.k_proj.weight', 'model.language_model.model.layers.3.self_attn.o_proj.weight', 'model.language_model.model.layers.3.self_attn.q_proj.weight', 'model.language_model.model.layers.3.self_attn.v_proj.weight', 'model.language_model.model.layers.4.input_layernorm.weight', 'model.language_model.model.layers.4.mlp.down_proj.weight', 'model.language_model.model.layers.4.mlp.gate_proj.weight', 'model.language_model.model.layers.4.mlp.up_proj.weight', 'model.language_model.model.layers.4.post_attention_layernorm.weight', 'model.language_model.model.layers.4.self_attn.k_proj.weight', 'model.language_model.model.layers.4.self_attn.o_proj.weight', 'model.language_model.model.layers.4.self_attn.q_proj.weight', 'model.language_model.model.layers.4.self_attn.v_proj.weight', 'model.language_model.model.layers.5.input_layernorm.weight', 'model.language_model.model.layers.5.mlp.down_proj.weight', 'model.language_model.model.layers.5.mlp.gate_proj.weight', 'model.language_model.model.layers.5.mlp.up_proj.weight', 'model.language_model.model.layers.5.post_attention_layernorm.weight', 'model.language_model.model.layers.5.self_attn.k_proj.weight', 'model.language_model.model.layers.5.self_attn.o_proj.weight', 'model.language_model.model.layers.5.self_attn.q_proj.weight', 'model.language_model.model.layers.5.self_attn.v_proj.weight', 'model.language_model.model.layers.6.input_layernorm.weight', 'model.language_model.model.layers.6.mlp.down_proj.weight', 'model.language_model.model.layers.6.mlp.gate_proj.weight', 'model.language_model.model.layers.6.mlp.up_proj.weight', 'model.language_model.model.layers.6.post_attention_layernorm.weight', 'model.language_model.model.layers.6.self_attn.k_proj.weight', 'model.language_model.model.layers.6.self_attn.o_proj.weight', 'model.language_model.model.layers.6.self_attn.q_proj.weight', 'model.language_model.model.layers.6.self_attn.v_proj.weight', 'model.language_model.model.layers.7.input_layernorm.weight', 'model.language_model.model.layers.7.mlp.down_proj.weight', 'model.language_model.model.layers.7.mlp.gate_proj.weight', 'model.language_model.model.layers.7.mlp.up_proj.weight', 'model.language_model.model.layers.7.post_attention_layernorm.weight', 'model.language_model.model.layers.7.self_attn.k_proj.weight', 'model.language_model.model.layers.7.self_attn.o_proj.weight', 'model.language_model.model.layers.7.self_attn.q_proj.weight', 'model.language_model.model.layers.7.self_attn.v_proj.weight', 'model.language_model.model.layers.8.input_layernorm.weight', 'model.language_model.model.layers.8.mlp.down_proj.weight', 'model.language_model.model.layers.8.mlp.gate_proj.weight', 'model.language_model.model.layers.8.mlp.up_proj.weight', 'model.language_model.model.layers.8.post_attention_layernorm.weight', 'model.language_model.model.layers.8.self_attn.k_proj.weight', 'model.language_model.model.layers.8.self_attn.o_proj.weight', 'model.language_model.model.layers.8.self_attn.q_proj.weight', 'model.language_model.model.layers.8.self_attn.v_proj.weight', 'model.language_model.model.layers.9.input_layernorm.weight', 'model.language_model.model.layers.9.mlp.down_proj.weight', 'model.language_model.model.layers.9.mlp.gate_proj.weight', 'model.language_model.model.layers.9.mlp.up_proj.weight', 'model.language_model.model.layers.9.post_attention_layernorm.weight', 'model.language_model.model.layers.9.self_attn.k_proj.weight', 'model.language_model.model.layers.9.self_attn.o_proj.weight', 'model.language_model.model.layers.9.self_attn.q_proj.weight', 'model.language_model.model.layers.9.self_attn.v_proj.weight', 'model.language_model.model.norm.weight', 'model.multi_modal_projector.linear.bias', 'model.multi_modal_projector.linear.weight', 'model.vision_tower.vision_model.embeddings.patch_embedding.bias', 'model.vision_tower.vision_model.embeddings.patch_embedding.weight', 'model.vision_tower.vision_model.embeddings.position_embedding.weight', 'model.vision_tower.vision_model.encoder.layers.0.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.0.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.0.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.0.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.0.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.0.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.0.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.0.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.1.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.1.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.1.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.1.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.1.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.1.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.1.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.1.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.10.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.10.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.10.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.10.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.10.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.10.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.10.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.10.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.11.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.11.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.11.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.11.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.11.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.11.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.11.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.11.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.12.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.12.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.12.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.12.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.12.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.12.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.12.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.12.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.13.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.13.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.13.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.13.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.13.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.13.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.13.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.13.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.14.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.14.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.14.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.14.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.14.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.14.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.14.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.14.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.15.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.15.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.15.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.15.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.15.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.15.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.15.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.15.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.16.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.16.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.16.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.16.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.16.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.16.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.16.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.16.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.17.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.17.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.17.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.17.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.17.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.17.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.17.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.17.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.18.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.18.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.18.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.18.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.18.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.18.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.18.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.18.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.19.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.19.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.19.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.19.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.19.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.19.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.19.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.19.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.2.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.2.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.2.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.2.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.2.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.2.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.2.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.2.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.20.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.20.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.20.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.20.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.20.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.20.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.20.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.20.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.21.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.21.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.21.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.21.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.21.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.21.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.21.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.21.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.22.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.22.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.22.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.22.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.22.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.22.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.22.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.22.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.23.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.23.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.23.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.23.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.23.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.23.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.23.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.23.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.24.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.24.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.24.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.24.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.24.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.24.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.24.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.24.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.24.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.24.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.24.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.24.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.24.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.24.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.24.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.24.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.25.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.25.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.25.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.25.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.25.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.25.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.25.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.25.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.25.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.25.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.25.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.25.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.25.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.25.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.25.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.25.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.26.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.26.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.26.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.26.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.26.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.26.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.26.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.26.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.26.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.26.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.26.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.26.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.26.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.26.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.26.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.26.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.3.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.3.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.3.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.3.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.3.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.3.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.3.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.3.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.4.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.4.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.4.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.4.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.4.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.4.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.4.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.4.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.5.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.5.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.5.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.5.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.5.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.5.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.5.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.5.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.6.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.6.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.6.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.6.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.6.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.6.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.6.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.6.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.7.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.7.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.7.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.7.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.7.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.7.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.7.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.7.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.8.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.8.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.8.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.8.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.8.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.8.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.8.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.8.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.weight', 'model.vision_tower.vision_model.encoder.layers.9.layer_norm1.bias', 'model.vision_tower.vision_model.encoder.layers.9.layer_norm1.weight', 'model.vision_tower.vision_model.encoder.layers.9.layer_norm2.bias', 'model.vision_tower.vision_model.encoder.layers.9.layer_norm2.weight', 'model.vision_tower.vision_model.encoder.layers.9.mlp.fc1.bias', 'model.vision_tower.vision_model.encoder.layers.9.mlp.fc1.weight', 'model.vision_tower.vision_model.encoder.layers.9.mlp.fc2.bias', 'model.vision_tower.vision_model.encoder.layers.9.mlp.fc2.weight', 'model.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.bias', 'model.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.weight', 'model.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.bias', 'model.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.weight', 'model.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.bias', 'model.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.weight', 'model.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.bias', 'model.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.weight', 'model.vision_tower.vision_model.post_layernorm.bias', 'model.vision_tower.vision_model.post_layernorm.weight']\n",
      "- This IS expected if you are initializing ColPali from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing ColPali from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of ColPali were not initialized from the model checkpoint at ./models/colpali-v1.3-merged and are newly initialized: ['model.lm_head.weight', 'model.model.language_model.embed_tokens.weight', 'model.model.language_model.layers.0.input_layernorm.weight', 'model.model.language_model.layers.0.mlp.down_proj.weight', 'model.model.language_model.layers.0.mlp.gate_proj.weight', 'model.model.language_model.layers.0.mlp.up_proj.weight', 'model.model.language_model.layers.0.post_attention_layernorm.weight', 'model.model.language_model.layers.0.self_attn.k_proj.weight', 'model.model.language_model.layers.0.self_attn.o_proj.weight', 'model.model.language_model.layers.0.self_attn.q_proj.weight', 'model.model.language_model.layers.0.self_attn.v_proj.weight', 'model.model.language_model.layers.1.input_layernorm.weight', 'model.model.language_model.layers.1.mlp.down_proj.weight', 'model.model.language_model.layers.1.mlp.gate_proj.weight', 'model.model.language_model.layers.1.mlp.up_proj.weight', 'model.model.language_model.layers.1.post_attention_layernorm.weight', 'model.model.language_model.layers.1.self_attn.k_proj.weight', 'model.model.language_model.layers.1.self_attn.o_proj.weight', 'model.model.language_model.layers.1.self_attn.q_proj.weight', 'model.model.language_model.layers.1.self_attn.v_proj.weight', 'model.model.language_model.layers.10.input_layernorm.weight', 'model.model.language_model.layers.10.mlp.down_proj.weight', 'model.model.language_model.layers.10.mlp.gate_proj.weight', 'model.model.language_model.layers.10.mlp.up_proj.weight', 'model.model.language_model.layers.10.post_attention_layernorm.weight', 'model.model.language_model.layers.10.self_attn.k_proj.weight', 'model.model.language_model.layers.10.self_attn.o_proj.weight', 'model.model.language_model.layers.10.self_attn.q_proj.weight', 'model.model.language_model.layers.10.self_attn.v_proj.weight', 'model.model.language_model.layers.11.input_layernorm.weight', 'model.model.language_model.layers.11.mlp.down_proj.weight', 'model.model.language_model.layers.11.mlp.gate_proj.weight', 'model.model.language_model.layers.11.mlp.up_proj.weight', 'model.model.language_model.layers.11.post_attention_layernorm.weight', 'model.model.language_model.layers.11.self_attn.k_proj.weight', 'model.model.language_model.layers.11.self_attn.o_proj.weight', 'model.model.language_model.layers.11.self_attn.q_proj.weight', 'model.model.language_model.layers.11.self_attn.v_proj.weight', 'model.model.language_model.layers.12.input_layernorm.weight', 'model.model.language_model.layers.12.mlp.down_proj.weight', 'model.model.language_model.layers.12.mlp.gate_proj.weight', 'model.model.language_model.layers.12.mlp.up_proj.weight', 'model.model.language_model.layers.12.post_attention_layernorm.weight', 'model.model.language_model.layers.12.self_attn.k_proj.weight', 'model.model.language_model.layers.12.self_attn.o_proj.weight', 'model.model.language_model.layers.12.self_attn.q_proj.weight', 'model.model.language_model.layers.12.self_attn.v_proj.weight', 'model.model.language_model.layers.13.input_layernorm.weight', 'model.model.language_model.layers.13.mlp.down_proj.weight', 'model.model.language_model.layers.13.mlp.gate_proj.weight', 'model.model.language_model.layers.13.mlp.up_proj.weight', 'model.model.language_model.layers.13.post_attention_layernorm.weight', 'model.model.language_model.layers.13.self_attn.k_proj.weight', 'model.model.language_model.layers.13.self_attn.o_proj.weight', 'model.model.language_model.layers.13.self_attn.q_proj.weight', 'model.model.language_model.layers.13.self_attn.v_proj.weight', 'model.model.language_model.layers.14.input_layernorm.weight', 'model.model.language_model.layers.14.mlp.down_proj.weight', 'model.model.language_model.layers.14.mlp.gate_proj.weight', 'model.model.language_model.layers.14.mlp.up_proj.weight', 'model.model.language_model.layers.14.post_attention_layernorm.weight', 'model.model.language_model.layers.14.self_attn.k_proj.weight', 'model.model.language_model.layers.14.self_attn.o_proj.weight', 'model.model.language_model.layers.14.self_attn.q_proj.weight', 'model.model.language_model.layers.14.self_attn.v_proj.weight', 'model.model.language_model.layers.15.input_layernorm.weight', 'model.model.language_model.layers.15.mlp.down_proj.weight', 'model.model.language_model.layers.15.mlp.gate_proj.weight', 'model.model.language_model.layers.15.mlp.up_proj.weight', 'model.model.language_model.layers.15.post_attention_layernorm.weight', 'model.model.language_model.layers.15.self_attn.k_proj.weight', 'model.model.language_model.layers.15.self_attn.o_proj.weight', 'model.model.language_model.layers.15.self_attn.q_proj.weight', 'model.model.language_model.layers.15.self_attn.v_proj.weight', 'model.model.language_model.layers.16.input_layernorm.weight', 'model.model.language_model.layers.16.mlp.down_proj.weight', 'model.model.language_model.layers.16.mlp.gate_proj.weight', 'model.model.language_model.layers.16.mlp.up_proj.weight', 'model.model.language_model.layers.16.post_attention_layernorm.weight', 'model.model.language_model.layers.16.self_attn.k_proj.weight', 'model.model.language_model.layers.16.self_attn.o_proj.weight', 'model.model.language_model.layers.16.self_attn.q_proj.weight', 'model.model.language_model.layers.16.self_attn.v_proj.weight', 'model.model.language_model.layers.17.input_layernorm.weight', 'model.model.language_model.layers.17.mlp.down_proj.weight', 'model.model.language_model.layers.17.mlp.gate_proj.weight', 'model.model.language_model.layers.17.mlp.up_proj.weight', 'model.model.language_model.layers.17.post_attention_layernorm.weight', 'model.model.language_model.layers.17.self_attn.k_proj.weight', 'model.model.language_model.layers.17.self_attn.o_proj.weight', 'model.model.language_model.layers.17.self_attn.q_proj.weight', 'model.model.language_model.layers.17.self_attn.v_proj.weight', 'model.model.language_model.layers.2.input_layernorm.weight', 'model.model.language_model.layers.2.mlp.down_proj.weight', 'model.model.language_model.layers.2.mlp.gate_proj.weight', 'model.model.language_model.layers.2.mlp.up_proj.weight', 'model.model.language_model.layers.2.post_attention_layernorm.weight', 'model.model.language_model.layers.2.self_attn.k_proj.weight', 'model.model.language_model.layers.2.self_attn.o_proj.weight', 'model.model.language_model.layers.2.self_attn.q_proj.weight', 'model.model.language_model.layers.2.self_attn.v_proj.weight', 'model.model.language_model.layers.3.input_layernorm.weight', 'model.model.language_model.layers.3.mlp.down_proj.weight', 'model.model.language_model.layers.3.mlp.gate_proj.weight', 'model.model.language_model.layers.3.mlp.up_proj.weight', 'model.model.language_model.layers.3.post_attention_layernorm.weight', 'model.model.language_model.layers.3.self_attn.k_proj.weight', 'model.model.language_model.layers.3.self_attn.o_proj.weight', 'model.model.language_model.layers.3.self_attn.q_proj.weight', 'model.model.language_model.layers.3.self_attn.v_proj.weight', 'model.model.language_model.layers.4.input_layernorm.weight', 'model.model.language_model.layers.4.mlp.down_proj.weight', 'model.model.language_model.layers.4.mlp.gate_proj.weight', 'model.model.language_model.layers.4.mlp.up_proj.weight', 'model.model.language_model.layers.4.post_attention_layernorm.weight', 'model.model.language_model.layers.4.self_attn.k_proj.weight', 'model.model.language_model.layers.4.self_attn.o_proj.weight', 'model.model.language_model.layers.4.self_attn.q_proj.weight', 'model.model.language_model.layers.4.self_attn.v_proj.weight', 'model.model.language_model.layers.5.input_layernorm.weight', 'model.model.language_model.layers.5.mlp.down_proj.weight', 'model.model.language_model.layers.5.mlp.gate_proj.weight', 'model.model.language_model.layers.5.mlp.up_proj.weight', 'model.model.language_model.layers.5.post_attention_layernorm.weight', 'model.model.language_model.layers.5.self_attn.k_proj.weight', 'model.model.language_model.layers.5.self_attn.o_proj.weight', 'model.model.language_model.layers.5.self_attn.q_proj.weight', 'model.model.language_model.layers.5.self_attn.v_proj.weight', 'model.model.language_model.layers.6.input_layernorm.weight', 'model.model.language_model.layers.6.mlp.down_proj.weight', 'model.model.language_model.layers.6.mlp.gate_proj.weight', 'model.model.language_model.layers.6.mlp.up_proj.weight', 'model.model.language_model.layers.6.post_attention_layernorm.weight', 'model.model.language_model.layers.6.self_attn.k_proj.weight', 'model.model.language_model.layers.6.self_attn.o_proj.weight', 'model.model.language_model.layers.6.self_attn.q_proj.weight', 'model.model.language_model.layers.6.self_attn.v_proj.weight', 'model.model.language_model.layers.7.input_layernorm.weight', 'model.model.language_model.layers.7.mlp.down_proj.weight', 'model.model.language_model.layers.7.mlp.gate_proj.weight', 'model.model.language_model.layers.7.mlp.up_proj.weight', 'model.model.language_model.layers.7.post_attention_layernorm.weight', 'model.model.language_model.layers.7.self_attn.k_proj.weight', 'model.model.language_model.layers.7.self_attn.o_proj.weight', 'model.model.language_model.layers.7.self_attn.q_proj.weight', 'model.model.language_model.layers.7.self_attn.v_proj.weight', 'model.model.language_model.layers.8.input_layernorm.weight', 'model.model.language_model.layers.8.mlp.down_proj.weight', 'model.model.language_model.layers.8.mlp.gate_proj.weight', 'model.model.language_model.layers.8.mlp.up_proj.weight', 'model.model.language_model.layers.8.post_attention_layernorm.weight', 'model.model.language_model.layers.8.self_attn.k_proj.weight', 'model.model.language_model.layers.8.self_attn.o_proj.weight', 'model.model.language_model.layers.8.self_attn.q_proj.weight', 'model.model.language_model.layers.8.self_attn.v_proj.weight', 'model.model.language_model.layers.9.input_layernorm.weight', 'model.model.language_model.layers.9.mlp.down_proj.weight', 'model.model.language_model.layers.9.mlp.gate_proj.weight', 'model.model.language_model.layers.9.mlp.up_proj.weight', 'model.model.language_model.layers.9.post_attention_layernorm.weight', 'model.model.language_model.layers.9.self_attn.k_proj.weight', 'model.model.language_model.layers.9.self_attn.o_proj.weight', 'model.model.language_model.layers.9.self_attn.q_proj.weight', 'model.model.language_model.layers.9.self_attn.v_proj.weight', 'model.model.language_model.norm.weight', 'model.model.multi_modal_projector.linear.bias', 'model.model.multi_modal_projector.linear.weight', 'model.model.vision_tower.vision_model.embeddings.patch_embedding.bias', 'model.model.vision_tower.vision_model.embeddings.patch_embedding.weight', 'model.model.vision_tower.vision_model.embeddings.position_embedding.weight', 'model.model.vision_tower.vision_model.encoder.layers.0.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.0.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.0.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.0.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.0.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.0.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.0.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.0.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.1.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.1.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.1.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.1.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.1.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.1.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.1.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.1.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.10.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.10.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.10.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.10.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.10.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.10.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.10.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.10.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.11.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.11.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.11.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.11.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.11.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.11.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.11.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.11.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.12.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.12.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.12.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.12.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.12.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.12.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.12.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.12.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.13.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.13.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.13.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.13.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.13.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.13.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.13.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.13.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.14.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.14.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.14.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.14.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.14.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.14.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.14.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.14.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.15.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.15.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.15.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.15.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.15.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.15.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.15.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.15.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.16.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.16.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.16.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.16.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.16.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.16.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.16.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.16.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.17.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.17.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.17.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.17.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.17.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.17.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.17.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.17.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.18.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.18.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.18.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.18.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.18.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.18.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.18.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.18.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.19.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.19.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.19.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.19.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.19.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.19.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.19.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.19.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.2.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.2.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.2.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.2.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.2.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.2.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.2.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.2.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.20.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.20.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.20.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.20.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.20.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.20.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.20.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.20.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.21.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.21.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.21.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.21.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.21.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.21.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.21.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.21.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.22.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.22.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.22.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.22.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.22.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.22.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.22.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.22.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.23.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.23.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.23.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.23.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.23.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.23.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.23.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.23.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.24.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.24.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.24.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.24.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.24.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.24.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.24.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.24.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.24.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.24.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.24.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.24.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.24.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.24.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.24.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.24.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.25.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.25.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.25.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.25.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.25.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.25.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.25.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.25.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.25.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.25.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.25.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.25.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.25.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.25.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.25.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.25.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.26.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.26.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.26.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.26.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.26.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.26.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.26.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.26.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.26.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.26.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.26.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.26.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.26.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.26.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.26.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.26.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.3.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.3.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.3.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.3.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.3.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.3.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.3.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.3.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.4.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.4.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.4.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.4.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.4.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.4.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.4.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.4.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.5.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.5.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.5.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.5.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.5.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.5.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.5.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.5.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.6.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.6.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.6.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.6.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.6.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.6.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.6.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.6.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.7.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.7.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.7.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.7.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.7.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.7.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.7.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.7.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.8.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.8.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.8.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.8.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.8.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.8.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.8.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.8.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.9.layer_norm1.bias', 'model.model.vision_tower.vision_model.encoder.layers.9.layer_norm1.weight', 'model.model.vision_tower.vision_model.encoder.layers.9.layer_norm2.bias', 'model.model.vision_tower.vision_model.encoder.layers.9.layer_norm2.weight', 'model.model.vision_tower.vision_model.encoder.layers.9.mlp.fc1.bias', 'model.model.vision_tower.vision_model.encoder.layers.9.mlp.fc1.weight', 'model.model.vision_tower.vision_model.encoder.layers.9.mlp.fc2.bias', 'model.model.vision_tower.vision_model.encoder.layers.9.mlp.fc2.weight', 'model.model.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.weight', 'model.model.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.bias', 'model.model.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.weight', 'model.model.vision_tower.vision_model.post_layernorm.bias', 'model.model.vision_tower.vision_model.post_layernorm.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n"
     ]
    }
   ],
   "source": [
    "# 加载模型（替换为ColPali，移除base_model_path参数）\n",
    "model = ColPali.from_pretrained(\n",
    "    local_model_path,\n",
    "    torch_dtype=torch.bfloat16,\n",
    "    device_map=\"cuda:0\",  # 无GPU可改为\"cpu\"\n",
    "    attn_implementation=\"flash_attention_2\" if is_flash_attn_2_available() else None,\n",
    "    local_files_only=True,  # 强制使用本地文件\n",
    "    trust_remote_code=True  # 必需，colpali含自定义模型代码\n",
    ").eval()\n",
    "\n",
    "# 加载处理器（替换为ColPaliProcessor，无需base_model_path）\n",
    "processor = ColPaliProcessor.from_pretrained(\n",
    "    local_model_path,\n",
    "    local_files_only=True  # 强制使用本地文件\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d441c5e13e1e7520",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T06:17:29.670844Z",
     "start_time": "2025-07-08T06:17:29.665928Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['T_destination', '__annotations__', '__call__', '__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattr__', '__getattribute__', '__getstate__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__setstate__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', '_apply', '_auto_class', '_autoset_attn_implementation', '_backward_compatibility_gradient_checkpointing', '_backward_hooks', '_backward_pre_hooks', '_buffers', '_call_impl', '_check_and_enable_flash_attn_2', '_check_and_enable_flash_attn_3', '_check_and_enable_flex_attn', '_check_and_enable_sdpa', '_checkpoint_conversion_mapping', '_compiled_call_impl', '_convert_head_mask_to_5d', '_copy_lm_head_original_to_resized', '_create_repo', '_device_mesh', '_dispatch_accelerate_model', '_fix_state_dict_key_on_load', '_fix_state_dict_key_on_save', '_fix_state_dict_keys_on_save', '_forward_hooks', '_forward_hooks_always_called', '_forward_hooks_with_kwargs', '_forward_pre_hooks', '_forward_pre_hooks_with_kwargs', '_from_config', '_get_backward_hooks', '_get_backward_pre_hooks', '_get_files_timestamps', '_get_key_renaming_mapping', '_get_name', '_get_no_split_modules', '_get_resized_embeddings', '_get_resized_lm_head', '_hf_peft_config_loaded', '_hook_rss_memory_post_forward', '_hook_rss_memory_pre_forward', '_init_added_embeddings_weights_with_mean', '_init_added_lm_head_bias_with_mean', '_init_added_lm_head_weights_with_mean', '_init_weights', '_initialize_missing_keys', '_initialize_weights', '_is_full_backward_hook', '_is_hf_initialized', '_is_stateful', '_keep_in_fp32_modules', '_keep_in_fp32_modules', '_keep_in_fp32_modules_strict', '_keep_in_fp32_modules_strict', '_keys_to_ignore_on_load_missing', '_keys_to_ignore_on_load_unexpected', '_keys_to_ignore_on_save', '_load_from_flax', '_load_from_state_dict', '_load_from_tf', '_load_pretrained_model', '_load_state_dict_post_hooks', '_load_state_dict_pre_hooks', '_maybe_warn_non_full_backward_hook', '_modules', '_move_missing_keys_from_meta_to_cpu', '_named_members', '_no_split_modules', '_no_split_modules', '_non_persistent_buffers_set', '_parameters', '_pp_plan', '_pp_plan', '_register_load_state_dict_pre_hook', '_register_state_dict_hook', '_replicate_for_data_parallel', '_resize_token_embeddings', '_save_to_state_dict', '_set_default_torch_dtype', '_set_gradient_checkpointing', '_skip_keys_device_placement', '_slow_forward', '_state_dict_hooks', '_state_dict_pre_hooks', '_supports_attention_backend', '_supports_cache_class', '_supports_flash_attn_2', '_supports_flash_attn_3', '_supports_flex_attn', '_supports_quantized_cache', '_supports_sdpa', '_supports_static_cache', '_tie_encoder_decoder_weights', '_tie_or_clone_weights', '_tied_weights_keys', '_tp_plan', '_tp_plan', '_tp_size', '_tp_size', '_upload_modified_files', '_version', '_wrapped_call_impl', 'active_adapter', 'active_adapters', 'add_adapter', 'add_memory_hooks', 'add_model_tags', 'add_module', 'apply', 'base_model', 'base_model_prefix', 'bfloat16', 'buffers', 'call_super_init', 'can_generate', 'children', 'compile', 'config', 'config_class', 'cpu', 'create_extended_attention_mask_for_decoder', 'cuda', 'custom_text_proj', 'delete_adapter', 'dequantize', 'device', 'dim', 'disable_adapters', 'disable_input_require_grads', 'double', 'dtype', 'dummy_inputs', 'dump_patches', 'enable_adapters', 'enable_input_require_grads', 'estimate_tokens', 'eval', 'extra_repr', 'float', 'floating_point_ops', 'forward', 'framework', 'from_pretrained', 'generation_config', 'get_adapter_state_dict', 'get_buffer', 'get_compiled_call', 'get_decoder', 'get_extended_attention_mask', 'get_extra_state', 'get_head_mask', 'get_init_context', 'get_input_embeddings', 'get_memory_footprint', 'get_output_embeddings', 'get_parameter', 'get_parameter_or_buffer', 'get_position_embeddings', 'get_submodule', 'gradient_checkpointing_disable', 'gradient_checkpointing_enable', 'half', 'hf_device_map', 'init_weights', 'initialize_weights', 'invert_attention_mask', 'ipu', 'is_backend_compatible', 'is_gradient_checkpointing', 'is_parallelizable', 'load_adapter', 'load_state_dict', 'loss_function', 'loss_type', 'main_input_name', 'mask_non_image_embeddings', 'model', 'model_tags', 'modules', 'mtia', 'name_or_path', 'named_buffers', 'named_children', 'named_modules', 'named_parameters', 'num_parameters', 'parameters', 'patch_size', 'post_init', 'prune_heads', 'push_to_hub', 'register_backward_hook', 'register_buffer', 'register_for_auto_class', 'register_forward_hook', 'register_forward_pre_hook', 'register_full_backward_hook', 'register_full_backward_pre_hook', 'register_load_state_dict_post_hook', 'register_load_state_dict_pre_hook', 'register_module', 'register_parameter', 'register_state_dict_post_hook', 'register_state_dict_pre_hook', 'requires_grad_', 'reset_memory_hooks_state', 'resize_position_embeddings', 'resize_token_embeddings', 'retrieve_modules_from_names', 'reverse_bettertransformer', 'save_pretrained', 'set_adapter', 'set_decoder', 'set_extra_state', 'set_input_embeddings', 'set_output_embeddings', 'set_submodule', 'share_memory', 'smart_apply', 'state_dict', 'supports_gradient_checkpointing', 'supports_pp_plan', 'supports_tp_plan', 'tie_weights', 'to', 'to_bettertransformer', 'to_empty', 'tp_size', 'train', 'training', 'type', 'warn_if_padding_and_no_attention_mask', 'warnings_issued', 'xpu', 'zero_grad']\n",
      "['__abstractmethods__', '__annotations__', '__call__', '__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__getstate__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__slots__', '__str__', '__subclasshook__', '__weakref__', '_abc_impl', '_auto_class', '_check_special_mm_tokens', '_create_repo', '_get_arguments_from_pretrained', '_get_files_timestamps', '_get_num_multimodal_tokens', '_merge_kwargs', '_upload_modified_files', 'apply_chat_template', 'attributes', 'audio_tokenizer', 'batch_decode', 'chat_template', 'check_argument_for_proper_class', 'create_plaid_index', 'decode', 'feature_extractor_class', 'from_args_and_dict', 'from_pretrained', 'get_image_mask', 'get_n_patches', 'get_possibly_dynamic_module', 'get_processor_dict', 'get_topk_plaid', 'image_processor', 'image_processor_class', 'image_seq_length', 'image_token', 'image_token_id', 'model_input_names', 'optional_attributes', 'optional_call_args', 'post_process_image_text_to_text', 'prepare_and_validate_optional_call_args', 'process_images', 'process_queries', 'process_texts', 'push_to_hub', 'query_augmentation_token', 'query_prefix', 'register_for_auto_class', 'save_pretrained', 'score', 'score_multi_vector', 'score_single_vector', 'to_dict', 'to_json_file', 'to_json_string', 'tokenizer', 'tokenizer_class', 'validate_init_kwargs', 'visual_prompt_prefix']\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# 查看模型支持的方法\n",
    "print(dir(model))  # 输出模型所有可用方法\n",
    "# 检查processor是否提供高级功能\n",
    "print(dir(processor))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "db16844990f35bca",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T06:17:34.140033Z",
     "start_time": "2025-07-08T06:17:34.092596Z"
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "# Your inputs\n",
    "images = [\n",
    "    Image.new(\"RGB\", (128, 128), color=\"white\"),\n",
    "    Image.new(\"RGB\", (64, 32), color=\"black\"),\n",
    "]\n",
    "queries = [\n",
    "    \"What is the organizational structure for our R&D department?\",\n",
    "    \"Can you provide a breakdown of last year’s financial performance?\",\n",
    "]\n",
    "\n",
    "\n",
    "# Process the inputs\n",
    "batch_images = processor.process_images(images).to(model.device)\n",
    "batch_queries = processor.process_queries(queries).to(model.device)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d16e80ceec6b834",
   "metadata": {},
   "source": [
    "### 快速开始"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a86b777875daded4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T06:17:43.734343Z",
     "start_time": "2025-07-08T06:17:38.454396Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[4.0000, 3.1094],\n",
      "        [4.5938, 3.3906]])\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Forward pass\n",
    "with torch.no_grad():\n",
    "    image_embeddings = model(**batch_images)\n",
    "    query_embeddings = model(**batch_queries)\n",
    "\n",
    "scores = processor.score_multi_vector(query_embeddings, image_embeddings)\n",
    "print(scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa01bd751386201a",
   "metadata": {},
   "source": [
    "### 热力图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "b5de5a8a56945a58",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T06:48:25.029944Z",
     "start_time": "2025-07-08T06:48:17.316346Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "torch.cuda.empty_cache()  # 释放未被引用的显存\n",
    "torch.cuda.ipc_collect()  # 清理跨进程缓存（可选）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "eeeb6b4f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T06:39:29.014052Z",
     "start_time": "2025-07-08T06:39:28.974473Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SiglipImageProcessor {\n",
      "  \"do_convert_rgb\": null,\n",
      "  \"do_normalize\": true,\n",
      "  \"do_rescale\": true,\n",
      "  \"do_resize\": true,\n",
      "  \"image_mean\": [\n",
      "    0.5,\n",
      "    0.5,\n",
      "    0.5\n",
      "  ],\n",
      "  \"image_processor_type\": \"SiglipImageProcessor\",\n",
      "  \"image_seq_length\": 1024,\n",
      "  \"image_std\": [\n",
      "    0.5,\n",
      "    0.5,\n",
      "    0.5\n",
      "  ],\n",
      "  \"processor_class\": \"ColPaliProcessor\",\n",
      "  \"resample\": 3,\n",
      "  \"rescale_factor\": 0.00392156862745098,\n",
      "  \"size\": {\n",
      "    \"height\": 448,\n",
      "    \"width\": 448\n",
      "  }\n",
      "}\n",
      "\n",
      "{'height': 448, 'width': 448}\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'SiglipImageProcessor' object has no attribute 'patch_size'",
     "output_type": "error",
     "traceback": [
      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
      "\u001b[31mAttributeError\u001b[39m                            Traceback (most recent call last)",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[28]\u001b[39m\u001b[32m, line 3\u001b[39m\n\u001b[32m      1\u001b[39m \u001b[38;5;28mprint\u001b[39m(processor.image_processor)\n\u001b[32m      2\u001b[39m \u001b[38;5;28mprint\u001b[39m(processor.image_processor.size)  \u001b[38;5;66;03m# 目标resize尺寸\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m3\u001b[39m \u001b[38;5;28mprint\u001b[39m(processor.image_processor.patch_size)  \u001b[38;5;66;03m# patch大小\u001b[39;00m\n\u001b[32m      4\u001b[39m \u001b[38;5;28mprint\u001b[39m(batch_images.shape)\n\u001b[32m      5\u001b[39m \u001b[38;5;28mprint\u001b[39m(image_embeddings.shape)\n",
      "\u001b[31mAttributeError\u001b[39m: 'SiglipImageProcessor' object has no attribute 'patch_size'"
     ]
    }
   ],
   "source": [
    "print(processor.image_processor)\n",
    "print(processor.image_processor.size)  # 目标resize尺寸\n",
    "print(processor.image_processor.patch_size)  # patch大小\n",
    "print(batch_images.shape)\n",
    "print(image_embeddings.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "5ae30d91ec228ef5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T06:41:18.918639Z",
     "start_time": "2025-07-08T06:41:18.856795Z"
    }
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "The number of patches (16 x 16 = 256) does not match the number of non-padded image tokens (1024).",
     "output_type": "error",
     "traceback": [
      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
      "\u001b[31mValueError\u001b[39m                                Traceback (most recent call last)",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[29]\u001b[39m\u001b[32m, line 7\u001b[39m\n\u001b[32m      4\u001b[39m image_mask = processor.get_image_mask(batch_images)\n\u001b[32m      6\u001b[39m \u001b[38;5;66;03m# 生成相似度热图（返回每个图像的热图）\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m7\u001b[39m batched_similarity_maps = get_similarity_maps_from_embeddings(\n\u001b[32m      8\u001b[39m     image_embeddings=image_embeddings,\n\u001b[32m      9\u001b[39m     query_embeddings=query_embeddings,\n\u001b[32m     10\u001b[39m     n_patches=n_patches,  \u001b[38;5;66;03m# 图像被分割的 patch 数量（如 16×16=256）\u001b[39;00m\n\u001b[32m     11\u001b[39m     image_mask=image_mask,\n\u001b[32m     12\u001b[39m )\n\u001b[32m     15\u001b[39m \u001b[38;5;66;03m# 可视化单个图像的热图\u001b[39;00m\n\u001b[32m     16\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mvisualize_heatmap\u001b[39m(image, heatmap, alpha=\u001b[32m0.6\u001b[39m):\n",
      "\u001b[36mFile \u001b[39m\u001b[32mD:\\ProgramData\\miniconda3\\envs\\colpali\\Lib\\site-packages\\colpali_engine\\interpretability\\similarity_map_utils.py:36\u001b[39m, in \u001b[36mget_similarity_maps_from_embeddings\u001b[39m\u001b[34m(image_embeddings, query_embeddings, n_patches, image_mask)\u001b[39m\n\u001b[32m     33\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m idx \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(image_embeddings.size(\u001b[32m0\u001b[39m)):\n\u001b[32m     34\u001b[39m     \u001b[38;5;66;03m# Sanity check\u001b[39;00m\n\u001b[32m     35\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m image_mask[idx].sum() != n_patches[idx][\u001b[32m0\u001b[39m] * n_patches[idx][\u001b[32m1\u001b[39m]:\n\u001b[32m---> \u001b[39m\u001b[32m36\u001b[39m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[32m     37\u001b[39m             \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mThe number of patches (\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mn_patches[idx][\u001b[32m0\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m x \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mn_patches[idx][\u001b[32m1\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m = \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m     38\u001b[39m             \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mn_patches[idx][\u001b[32m0\u001b[39m]\u001b[38;5;250m \u001b[39m*\u001b[38;5;250m \u001b[39mn_patches[idx][\u001b[32m1\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m) \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m     39\u001b[39m             \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mdoes not match the number of non-padded image tokens (\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mimage_mask[idx].sum()\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m).\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m     40\u001b[39m         )\n\u001b[32m     42\u001b[39m     \u001b[38;5;66;03m# Rearrange the output image tensor to explicitly represent the 2D grid of patches\u001b[39;00m\n\u001b[32m     43\u001b[39m     image_embedding_grid = rearrange(\n\u001b[32m     44\u001b[39m         image_embeddings[idx][image_mask[idx]],  \u001b[38;5;66;03m# (n_patches_x * n_patches_y, dim)\u001b[39;00m\n\u001b[32m     45\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33m(h w) c -> w h c\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m     46\u001b[39m         w=n_patches[idx][\u001b[32m0\u001b[39m],\n\u001b[32m     47\u001b[39m         h=n_patches[idx][\u001b[32m1\u001b[39m],\n\u001b[32m     48\u001b[39m     )  \u001b[38;5;66;03m# (n_patches_x, n_patches_y, dim)\u001b[39;00m\n",
      "\u001b[31mValueError\u001b[39m: The number of patches (16 x 16 = 256) does not match the number of non-padded image tokens (1024)."
     ]
    }
   ],
   "source": [
    "n_patches = (16, 16)\n",
    "\n",
    "# 获取图像的 patch 掩码（用于过滤无关区域）\n",
    "image_mask = processor.get_image_mask(batch_images)\n",
    "\n",
    "# 生成相似度热图（返回每个图像的热图）\n",
    "batched_similarity_maps = get_similarity_maps_from_embeddings(\n",
    "    image_embeddings=image_embeddings,\n",
    "    query_embeddings=query_embeddings,\n",
    "    n_patches=n_patches,  # 图像被分割的 patch 数量（如 16×16=256）\n",
    "    image_mask=image_mask,\n",
    ")\n",
    "\n",
    "\n",
    "# 可视化单个图像的热图\n",
    "def visualize_heatmap(image, heatmap, alpha=0.6):\n",
    "    plt.figure(figsize=(10, 10))\n",
    "    plt.imshow(image)  # 显示原始图像\n",
    "    plt.imshow(heatmap, cmap='jet', alpha=alpha)  # 叠加热图\n",
    "    plt.colorbar()  # 显示颜色刻度\n",
    "    plt.axis('off')\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "# 示例：可视化第一张图像的热图\n",
    "visualize_heatmap(\n",
    "    image=images[0],  # PIL 图像\n",
    "    heatmap=batched_similarity_maps[0].reshape(n_patches, n_patches),\n",
    "    alpha=0.6  # 热图透明度\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
